#!/usr/bin/env python

#
#    pywget.py - downloading files via wget
#

from subprocess import call
import sys

__version="0.1"

def wget(url, params):
    """
    Download file via wget
    Return:
        Done        - if file downloaded
        Failed      - some wget/network problems
        Interrupted - user interrupt
    """
    if 'wget_parameters' in params.keys():
        run_wget = "wget %s %s" % (params['wget_parameters'], url)
    else:
        run_wget = "wget %s %s" % ('', url)
    try:
        retcode = call(run_wget, shell=True)
        print "Return code: %d" % retcode
        if retcode < 0:
            return "Interruped"
        elif retcode == 0:
            return "Done"
        else:
            return "Failed"
    except KeyboardInterrupt:
        return "Interrupted"

def wget_list(filename, params):
    """
    Download files from file via wget
    """
    try:
        urls = open(filename, 'r').readlines()
    except IOError, err:
        print err
        sys.exit()

    try:        
        for i, url in enumerate(urls):
            if url[-1] == "\n": url = url[:-1]
            if not url: continue
            if (url.find(':downloaded') > 0) or (url.find(':failed') > 0): continue
            
            retcode = wget(url, params)
            
            if retcode == 'Done':
                print "Download comleted"
                urls[i] = "%s:downloaded\n" % url
            elif retcode == 'Failed':
                print "Download failed"
                urls[i] = "%s:failed\n" % url
            elif retcode == 'Interrupted':
                break
    finally:
        open(filename, 'w').writelines(urls)

def wget_db_select(cursor, sql):
    """
    Extract info from database
    """
    data = []
    cursor.execute(sql)
    for row in cursor:
        data.append(row)
    return data

def wget_db_update(cursor, sql):
    """
    update data in database
    """
    cursor.execute(sql)
    
    
def wget_db(database, params):
    """
    Download files from database via wget
    """
    
    if 'sqlite_urls_extract' not in params.keys() \
        or 'sqlite_url_ok_status_update' not in params.keys() \
        or 'sqlite_url_not_ok_status_update' not in params.keys(): 
        
        print '\nError! For database handling the next parameters are mandatory in config file:'
        print '\tsqlite:urs_extract'
        print '\tsqlite:url_ok_status_update'
        print '\tsqlite:url_not_ok_status_update'
        sys.exit()
        
    import sqlite3
    conn = sqlite3.connect(database)
    cursor = conn.cursor()
    while True:
        try:
            url = wget_db_select(cursor, params['sqlite_urls_extract'])[0][0]
            if url:
                retcode = wget(url, params)
                
                if retcode == 'Done':
                    wget_db_update(cursor, params['sqlite_url_ok_status_update'] % url)
                    conn.commit()
                if retcode == 'Failed':
                    print '****** Failed'
                    wget_db_update(cursor, params['sqlite_url_not_ok_status_update'] % url)
                    conn.commit()
                elif retcode == 'Interrupted':
                    break
            else:
                break
        except:
            break
    
def wget_params(filename):
    """
    Load wget params
    """
    import ConfigParser
    config = ConfigParser.ConfigParser()
    config.read(filename)
    params = {}
    # wget section
    try:
        params['wget_parameters'] = config.get('wget', 'parameters') 
    except ConfigParser.NoSectionError:
        pass
    except ConfigParser.NoOptionError:
        pass
    # sqlite section
    try:
        params['sqlite_urls_extract'] = config.get('sqlite', 'urls_extract') 
        params['sqlite_url_ok_status_update'] = config.get('sqlite', 'url_ok_status_update') 
        params['sqlite_url_not_ok_status_update'] = config.get('sqlite', 'url_not_ok_status_update') 
    except ConfigParser.NoSectionError:
        pass
    except ConfigParser.NoOptionError:
        pass
    return params

if __name__ == "__main__":
    from optparse import OptionParser
    parser = OptionParser(version="%prog, v." + __version)
    parser.add_option("-c", "--config", dest="config", help="configuration file")
    parser.add_option("-u", "--url", dest="url", help="URL")
    parser.add_option("-l", "--url_list", dest="url_list", help="file with list of URLs")
    parser.add_option("-d", "--url_database", dest="url_database", 
                        help="sqlite database with list of URLs, please use config file for handling urls")
    (options, args) = parser.parse_args()

    params = {}
    # default parameters for wget
    params['wget_parameters'] = '-c'
    
    if options.config != None:
        print "Config file: %s" % options.config
        params = wget_params(options.config)

    if options.url != None:
        print "URL: %s" % options.url 
        if wget(options.url, params):
            print "Download comleted"
        else:
            print "Download failed"
            
    elif options.url_list != None:
        print "Getting URLs from file: %s" % options.url_database
        wget_list(options.url_list, params)
        
    elif options.url_database != None:
        print "Getting URLs from sqlite database: %s" % options.url_database
        wget_db(options.url_database, params)
        
    else:
        parser.error("Please specify URL or database parameter. \n\nuse -h or --help for detail")
            
        
